﻿using DotnetSpider.Core;
using DotnetSpider.Core.Pipeline;
using DotnetSpider.Core.Processor;
using DotnetSpider.Core.Scheduler;
using DotnetSpider.Core.Selector;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;

namespace AreaSpider
{
    class MainSpider
    {

        static List<Area> areaLst = new List<Area>();
        static string baseUrl = "http://www.mca.gov.cn";
        static string startUrl = baseUrl + $"/article/sj/tjbz/a/{DateTime.Now.Year}/";

        public static void Main()
        {
            // 定义要采集的 Site 对象, 可以设置 Header、Cookie、代理等
            var site = new Site { EncodingName = "UTF-8", RemoveOutboundLinks = true };
            site.AddStartUrl(startUrl);
            Spider spider = Spider.Create(site, new QueueDuplicateRemovedScheduler(), new StartPageProcessor()).AddPipeline(new StartPipeLine()).SetThreadNum(1);
            spider.EmptySleepTime = 3000;
            spider.Deep = 2;

            // 启动爬虫
            spider.Run();
            Console.Read();
        }

        private class StartPipeLine : BasePipeline
        {

            public override void Process(ResultItems resultItems)
            {
                foreach (string url in resultItems.Results["UrlList"])
                {
                    //未找到有效的行政区划，继续，找到则退出
                    if (areaLst.Count > 0) break;
                    Console.ForegroundColor = ConsoleColor.Yellow;
                    Console.WriteLine($"找到链接 {url}");
                    var site = new Site { EncodingName = "UTF-8" };
                    site.AddStartUrl(url);
                    DotnetSpider.Core.Spider spider = DotnetSpider.Core.Spider.Create(site, new QueueDuplicateRemovedScheduler(), new ShtmlPageProcessor()).AddPipeline(new ShtmlPipeLine()).SetThreadNum(1);
                    spider.EmptySleepTime = 3000;
                    spider.Deep = 2;

                    //启动爬虫
                    spider.Run();

                }
            }
        }

        private class StartPageProcessor : BasePageProcessor
        {
            protected override void Handle(Page page)
            {
                // 利用 Selectable 查询并构造自己想要的数据对象
                var ulrTable = page.Selectable.Select(Selectors.XPath("//*[@id='list_main']/div[2]/div[2]/ul/table"));
                var trs = ulrTable.SelectList(Selectors.XPath("./tr")).Nodes();
                //Console.WriteLine(page.Url);
                List<string> urlList = new List<string>();
                foreach (var tr in trs)
                {
                    var tds = tr.SelectList(Selectors.XPath("./td")).Nodes();
                    if (tds.Count >= 2)
                    {
                        if (tds[1].Links() != null)
                        {
                            urlList.Add(tds[1].Links().GetValue());
                        }
                        page.AddResultItem("UrlList", urlList);

                    }
                }

            }
        }

        private class ShtmlPipeLine : BasePipeline
        {

            public override void Process(ResultItems resultItems)
            {
                string url = resultItems.Results["areaUrlList"];
                if ((areaLst == null || areaLst.Count == 0) && !string.IsNullOrEmpty(url))
                {
                    Console.ForegroundColor = ConsoleColor.Yellow;
                    Console.WriteLine($"找到行政区划链接 {url}");
                    var site = new Site { EncodingName = "UTF-8" };
                    site.AddStartUrl(url);
                    DotnetSpider.Core.Spider spider = DotnetSpider.Core.Spider.Create(site, new QueueDuplicateRemovedScheduler(), new AreaPageProcessor()).AddPipeline(new AreaPipeLine()).SetThreadNum(1);
                    spider.EmptySleepTime = 3000;
                    spider.Deep = 2;

                    //启动爬虫
                    spider.Run();
                }
            }
        }

        private class ShtmlPageProcessor : BasePageProcessor
        {
            protected override void Handle(Page page)
            {

                //找到真实页面
                var ulrTable = page.Selectable.Select(Selectors.XPath("//body/script"));
                //window.location.href = "/article/sj/tjbz/a/2016/20161010/201612021609.html";
                string script = ulrTable.GetValue().Trim();
                //爬真实页面
                if (script.StartsWith("window.location.href"))
                {
                    string url = baseUrl + script.Split('"')[1];

                    page.AddResultItem("areaUrlList", url);
                }

            }
        }

        private class AreaPipeLine : BasePipeline
        {

            public override void Process(ResultItems resultItems)
            {
                FileSet.SaveFile(resultItems.Results["AreaList"].areaLst, resultItems.Results["AreaList"].url);
            }
        }

        private class AreaPageProcessor : BasePageProcessor
        {
            protected override void Handle(Page page)
            {

                var trs = page.Selectable.SelectList(Selectors.XPath("//div[1]/table/tr")).Nodes();
                var tdTitle = trs[0].Select(Selectors.XPath("./td"));

                //是县以上行政区划发布才需要处理
                if (!tdTitle.GetValue().Contains("县以上"))
                {
                    Console.ForegroundColor = ConsoleColor.Yellow;
                    Console.WriteLine("非县以上行政区划发布信息，放弃，继续往下查找......");
                    return;
                }
                Console.ForegroundColor = ConsoleColor.Green;
                Console.WriteLine("找到县以上行政区划发布信息，开始处理......");
                for (int i = 3; i < trs.Count; i++)
                {
                    var tds = trs[i].SelectList(Selectors.XPath("./td")).Nodes();
                    if (tds.Count > 3 && !string.IsNullOrEmpty(tds[1].GetValue()) && tds[1].GetValue().Length > 0)
                    {
                        string code = tds[1].GetValue().Trim();
                        //名称只匹配中文
                        Match match = Regex.Match(tds[2].GetValue(), @"[\u4E00-\u9FA5]+");
                        string name = "";
                        foreach (Group g in match.Groups)
                        {
                            name = name + g.Value;
                        }

                        //Console.Read();                            
                        Area area = new Area()
                        {
                            Code = code,
                            Name = name
                        };
                        areaLst.Add(area);
                    }
                }
                page.AddResultItem("AreaList", new { areaLst = areaLst, url = page.Url });
            }
        }


    }

    public class Area
    {
        public string Code { get; set; }
        public string Name { get; set; }
    }
}
